﻿using CliWrap;
using CliWrap.EventStream;
using FreeSql;
using Ldt.Aop.Statistical;
using LdtCommonFreeSql;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using NlpApp.Model;
using NlpTrainApp.Bll.Model;
using NlpTrainApp.Bll.Model.Dao;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace NlpTrainApp.Bll.Uie
{
    /// <summary>
    /// uie训练业务类
    /// </summary>
    public class TrainServiceBll
    {
        private readonly string baseDataPath = Path.Combine(AppContext.BaseDirectory, "datasets", "DuIE");
        private readonly IConfiguration configuration;
        private readonly ExtractServiceBll extractService;
        private readonly FreesqlDb freesqlDb;
        private readonly HubMessageServer hubMessage;
        private readonly ILogger<TrainServiceBll> logger;
        private readonly IFreeSql freeSqlCaseCreate52;
        public TrainServiceBll(ILogger<TrainServiceBll> logger, FreesqlDb freesqlDb, IConfiguration configuration, HubMessageServer hubMessage, ExtractServiceBll extractServiceBll)
        {
            this.logger = logger;
            this.freesqlDb = freesqlDb;
            this.configuration = configuration;
            this.hubMessage = hubMessage;
            extractService = extractServiceBll;
            freeSqlCaseCreate52 = freesqlDb.GetFreeSql("casecausecreate", new ConnConfig
            {
                Dbcode = "casecausecreate",
                Dbtype = FreeSql.DataType.PostgreSQL,
                IP = "192.168.1.52",
                MaxPool = "1",
                Password = "ldtlaw@123",
                Port = "5432",
                Username = "ldt"
            });
        }

        public bool IsTrainIng { get; set; }
        /// <summary>
        /// Starts the train.docker run --ulimit core=0 -it -e NVIDIA_VISIBLE_DEVICES=0 -e NVIDIA_DRIVER_CAPABILITIES=compute,utility --runtime=nvidia --name imgclassifytrain -p 8001:8000 -p 9090:9090 -v /root/trainimg:/root/trainimg -v /home/trainimgapp:/home/trainimgapp imgclassify:8.6 bash
        /// </summary>
        /// <param name="incrementalLearning">if set to <c>true</c> [incremental learning].</param>
        /// <param name="code">The code.文书名称</param>
        /// <exception cref="System.Exception">没有新的训练样本数据</exception>
        [StatisticalTimeOnlyMethod(LogLevelItem = "Info", IsLogException = true)]
        public async Task StartTrain(bool incrementalLearning, string code)
        {
            try
            {
                Random rn = new Random();
                IsTrainIng = true;
                await hubMessage.SendMessage(new HubMessageModel { Content = $"开始构造训练数据" });
                //doccano导出数据后可通过doccano.py脚本轻松将数据转换为输入模型时需要的形式 ldt123456
                var dataModels = await CreateClsTrainData();
                dataModels.AddRange(await MergeTrainData(incrementalLearning, code));
                if (dataModels.Any())
                {
                    _ = hubMessage.SendMessage(new HubMessageModel { Content = $"训练数据{dataModels.Count}" });
                    File.WriteAllText(Path.Combine(baseDataPath, "train.json"), string.Join("\n", dataModels.Select(p => JsonConvert.SerializeObject(p))));
                    //File.Copy(Path.Combine(baseDataPath, "train.json"), Path.Combine(baseDataPath, "dev.json"), true);
                    int cc1 = Convert.ToInt32(dataModels.Count * 0.5);
                    File.WriteAllText(Path.Combine(baseDataPath, "dev.json"), string.Join("\n",
                        dataModels.OrderBy(p => rn.Next()).Take(cc1).Select(p => JsonConvert.SerializeObject(p))));
                    await hubMessage.SendMessage(new HubMessageModel { Content = $"构造训练数据完成" });
                    var uietrain = dataModels.Select(p => new uietrain
                    {
                        content = p.content,
                        prompt = p.prompt,
                        result_list = string.Join(",", p.Result_list.Select(a => a.Text))
                    }).ToList();
                    freeSqlCaseCreate52.Ado.ExecuteNonQuery("delete from uietrain");
                    freeSqlCaseCreate52.Insert(uietrain).ExecuteAffrows();
                    _ = RunTrain(incrementalLearning);
                }
                else
                {
                    throw new Exception("没有新的训练样本数据");
                }
            }
            catch (Exception ex)
            {
                logger.LogError(ex, "失败");
                _ = hubMessage.SendMessage(new HubMessageModel { Content = $"异常{ex.Message}{ex.StackTrace}" });
            }
            finally
            {
                IsTrainIng = false;
            }
        }
        /// <summary>
        /// Creates the CLS train data.合并所有的训练数据(分类)
        /// </summary>
        private async Task<List<TrainDataModel>> CreateClsTrainData()
        {
            List<TrainDataModel> dataModels = new List<TrainDataModel>();
            var freesql = freesqlDb.GetFreeSql("doccano", new ConnConfig
            {
                Dbtype = DataType.Sqlite,
                Dbcode = configuration["ConnectionString"]
            });
            var texts = await freesql.Select<Examples_example>()
                .Where(a => freesql.Select<Projects_project>().Where(p => p.Project_type == "DocumentClassification" && p.Id == a.Project_id).Any())
                .Where(a => freesql.Select<Examples_examplestate>().Where(p => p.Confirmed_by_id == 1 && a.Id == p.Example_id).Any())
                .ToListAsync();
            var labels = await freesql.Select<Label_types_spantype>().AsTable((a, oldname) => "label_types_categorytype").ToListAsync();
            var groupText = texts.GroupBy(p => p.Project_id);
            foreach (var group in groupText)
            {
                List<DoccanoClsTrainDataModel> trainDataModels = new List<DoccanoClsTrainDataModel>();
                var labelsGroup = labels.Select(p => new Label_types_spantype
                {
                    Id = p.Id,
                    Text = p.Text.Split('-')[1],
                    Prefix_key = p.Text.Split('-')[0],
                    Project_id = p.Project_id,
                }).Where(p => p.Project_id == group.Key).ToList();
                foreach (var item in group.ToList())
                {
                    var aa1 = await freesql.Select<Labels_span>().AsTable((a, b) => "labels_category").Where(p => p.Example_id == item.Id).ToListAsync(p => new Labels_span
                    {
                        Label_id = p.Label_id
                    });
                    var dd1 = new DoccanoClsTrainDataModel
                    {
                        Id = item.Id,
                        Text = item.Text,
                        Label = aa1.Select(p => labelsGroup.Find(a => a.Id == p.Label_id).Text).ToList()
                    };
                    trainDataModels.Add(dd1);
                }
                File.WriteAllText(Path.Combine(baseDataPath, "doccano_ext.json"), string.Join("\n", trainDataModels.Select(p => JsonConvert.SerializeObject(p))));
                await RunDoccanoDataConvert(false, labelsGroup[0].Prefix_key, string.Join(",", labelsGroup.Select(p => p.Text)));
                var datass = File.ReadAllLines(Path.Combine(baseDataPath, "train.json"));
                var objs = datass.Select(p => JsonConvert.DeserializeObject<TrainDataModel>(p)).Where(p => p != null).ToList();
                objs.ForEach(a =>
                {
                    a.Result_list = a.Result_list.Where(p => p.Text.Trim() != "").ToList();
                });
                dataModels.AddRange(objs);
            }
            return dataModels;
        }
        /// <summary>
        /// Merges the train data.合并所有的训练数据(实体、关系数据)
        /// </summary>
        private async Task<List<TrainDataModel>> MergeTrainData(bool incrementalLearning, string code)
        {
            List<TrainDataModel> dataModels = new List<TrainDataModel>();
            var freesql = freesqlDb.GetFreeSql("doccano", new ConnConfig
            {
                Dbtype = DataType.Sqlite,
                Dbcode = configuration["ConnectionString"]
            });
            var texts = await freesql.Select<Examples_example>()
                .Where(a => freesql.Select<Projects_project>().WhereIf(!string.IsNullOrWhiteSpace(code), p => p.Name.Contains(code)).Where(p => p.Project_type == "SequenceLabeling"
                                                                && p.Id == a.Project_id && !p.Name.Contains("负样本") && !p.Name.Contains("不训练")).Any())
                .Where(a => freesql.Select<Examples_examplestate>().Where(p => p.Confirmed_by_id == 1 && a.Id == p.Example_id).Any())
                .ToListAsync();
            var labels = await freesql.Select<Label_types_spantype>().ToListAsync();
            var projects = await freesql.Select<Projects_project>().ToListAsync();
            var releations = await freesql.Select<Label_types_relationtype>().ToListAsync();
            var groupText = texts.GroupBy(p => p.Project_id);
            logger.LogInformation($"项目:{string.Join(",", groupText.Select(p => p.Key).ToList())}");
            foreach (var group in groupText)
            {
                List<DoccanoTrainDataModel> trainDataModels = new List<DoccanoTrainDataModel>();
                var labelsGroup = labels.Where(p => p.Project_id == group.Key).ToList();
                foreach (var item in group.ToList())
                {
                    var meta = JsonConvert.DeserializeObject<DoccanoTrainDataModel>(item.Meta);
                    var aa1 = await freesql.Select<Labels_span>().Where(p => p.Example_id == item.Id).ToListAsync();
                    var bb1 = await freesql.Select<Labels_relation>().Where(p => p.Example_id == item.Id).ToListAsync();
                    var dd1 = new DoccanoTrainDataModel
                    {
                        Id = meta.Id,
                        Text = item.Text,
                        Entities = aa1.Select(p => new EntitiesItem
                        {
                            Id = p.Id,
                            Start_offset = p.Start_offset,
                            End_offset = p.End_offset,
                            Label = labelsGroup.First(k => k.Id == p.Label_id).Text
                        }).ToList(),
                        Relations = bb1.Select(p => new RelationsItem
                        {
                            Id = p.Id,
                            From_id = p.From_id_id,
                            To_id = p.To_id_id,
                            Type = releations.First(k => k.Id == p.Type_id).Text
                        }).ToList()
                    };
                    trainDataModels.Add(dd1);
                }
                File.WriteAllText(Path.Combine(baseDataPath, "doccano_ext.json"), string.Join("\n", trainDataModels.Select(p => JsonConvert.SerializeObject(p))));
                await RunDoccanoDataConvert(incrementalLearning || projects.Exists(p => p.Id == group.Key && p.Name.Contains("正样本")));
                var datass = File.ReadAllLines(Path.Combine(baseDataPath, "train.json"));
                var objs = datass.Select(p => JsonConvert.DeserializeObject<TrainDataModel>(p)).Where(p => p != null).ToList();
                objs.ForEach(a =>
                {
                    a.Result_list = a.Result_list.Where(p => p.Text.Trim() != "").ToList();
                });
                dataModels.AddRange(objs);
            }

            //负样本
            texts = await freesql.Select<Examples_example>()
    .Where(a => freesql.Select<Projects_project>().Where(p => p.Project_type == "SequenceLabeling" && p.Id == a.Project_id
                                                                    && p.Name.Contains("负样本") && !p.Name.Contains("不训练")).Any())
    .Where(a => freesql.Select<Examples_examplestate>().Where(p => p.Confirmed_by_id == 1 && a.Id == p.Example_id).Any())
    .ToListAsync();
            groupText = texts.GroupBy(p => p.Project_id);
            foreach (var group in groupText)
            {
                var labelsGroup = labels.Where(p => p.Project_id == group.Key).ToList();
                foreach (var item in group.ToList())
                {
                    var aa1 = await freesql.Select<Labels_span>().Where(p => p.Example_id == item.Id).ToListAsync();
                    dataModels.AddRange(aa1.Select(p => new TrainDataModel
                    {
                        content = item.Text,
                        prompt = labelsGroup.First(k => k.Id == p.Label_id).Text,
                    }));
                }
            }
            //手动的负样本
            var ttt1 = freeSqlCaseCreate52.Select<TrainDataModel>().AsTable((a, b) => configuration["NegTableName"] ?? "uieerror").ToList(p => new TrainDataModel
            {
                content = p.content,
                prompt = p.prompt
            });
            dataModels.AddRange(ttt1);
            return dataModels;
        }
        /// <summary>
        ///负样本的生成要只在单个数据集中 doccano python doccano.py --doccano_file ./data/doccano_ext.json --task_type ext --save_dir ./data --splits 0.8 0.2 0 --schema_lang ch
        /// </summary>
        /// <param name="prompt_prefix">是否分类 不为空就是分类前置信息</param>
        /// <param name="options">是否分类 不为空就是分类类别</param>
        /// <returns></returns>
        private async Task RunDoccanoDataConvert(bool incrementalLearning = false, string prompt_prefix = null, string options = null)
        {
            var cmd = Cli.Wrap("python").WithWorkingDirectory(AppContext.BaseDirectory);
            if (string.IsNullOrEmpty(prompt_prefix))
            {
                if (incrementalLearning)
                {
                    cmd = cmd.WithArguments(new[] { "doccano.py", "--doccano_file", Path.Combine(baseDataPath, "doccano_ext.json"), "--task_type", "ext", "--save_dir", baseDataPath, "--splits", "1", "0", "0", "--schema_lang", "ch", "--negative_ratio", "0"});
                }
                else
                {
                    cmd = cmd.WithArguments(new[] { "doccano.py", "--doccano_file", Path.Combine(baseDataPath, "doccano_ext.json"), "--task_type", "ext", "--save_dir", baseDataPath, "--splits", "1", "0", "0", "--schema_lang", "ch" });
                }
            }
            else
            {
                cmd = cmd.WithArguments(new[] { "doccano.py", "--doccano_file", Path.Combine(baseDataPath, "doccano_ext.json"), "--task_type", "cls", "--save_dir", baseDataPath, "--splits", "1", "0", "0", "--schema_lang", "ch",
                "--prompt_prefix", $"{prompt_prefix}","--options", $"{options}"
                        });
            }
            await foreach (var cmdEvent in cmd.ListenAsync())
            {
                switch (cmdEvent)
                {
                    case StartedCommandEvent:
                        break;
                    case ExitedCommandEvent exited:
                        logger.LogInformation($"RunDoccanoDataConvert完成{exited}");
                        break;
                    case StandardErrorCommandEvent err:
                        logger.LogInformation($"RunDoccanoDataConvert异常{err}");
                        break;
                    default:
                        logger.LogInformation($"{cmdEvent}");
                        break;
                }
            }
        }

        /// <summary>
        /// Runs the train.
        /// </summary>
        private async Task RunTrain(bool incrementalLearning)
        {
            _ = hubMessage.SendMessage(new HubMessageModel { Content = $"开始训练" });
            //var cmd = Cli.Wrap("python").WithWorkingDirectory(AppContext.BaseDirectory);
            //cmd = cmd.WithArguments(new[] { "train.py","--pretrained_model", "uie-base-zh",
            //    "--save_dir", "checkpoints/DuIE","--train_path",Path.Combine(baseDataPath,"train.txt"),
            //    "--dev_path",Path.Combine(baseDataPath, "dev.txt"),"--img_log_dir","logs/","--img_log_name",
            //    "UIE Base","--batch_size", "1","--max_seq_len", "512","--learning_rate","5e-5",
            //    "--num_train_epochs", "10","--logging_steps" ,"1","--device", configuration["Device"] });
            var cmd = Cli.Wrap("bash").WithWorkingDirectory(AppContext.BaseDirectory);
            cmd = cmd.WithArguments(new[] { incrementalLearning ? "litetrain.sh" : "litetrainbase.sh" });
            await foreach (var cmdEvent in cmd.ListenAsync())
            {
                switch (cmdEvent)
                {
                    case StartedCommandEvent:
                        break;
                    case ExitedCommandEvent exited:
                        logger.LogInformation($"RunTrain完成{exited}");
                        LdtCommonUtil.FileIO.FileUtils.CopyPath(Path.Combine(AppContext.BaseDirectory, "uie_lit_model_pytorch", "ernie-uie"), Path.Combine(AppContext.BaseDirectory, "uie_lit_model_pytorch"));
                        _ = hubMessage.SendMessage(new HubMessageModel { Content = $"RunTrain完成" });
                        _ = extractService.RestartClassify();
                        break;
                    case StandardErrorCommandEvent err:
                        logger.LogInformation($"{err}");
                        break;
                    default:
                        logger.LogInformation($"{cmdEvent}");
                        _ = hubMessage.SendMessage(new HubMessageModel { Content = $"{cmdEvent}" });
                        break;
                }
            }
            logger.LogInformation($"训练完成");
            _ = hubMessage.SendMessage(new HubMessageModel { Content = $"训练完成" });
        }
    }
}
